# -*- coding: utf-8 -*-
import re,os,sqlite3,urllib2,datetime,zlib,gzip
from pyquery import PyQuery as pq
# conn=sqlite3.connect('merge_err.db3')
conn=sqlite3.connect('cif_company.db3')
conn.text_factory = str

def htmlclean(html):
    # 多于1个的换行、tab和&nbsp;，替换为一个空格
    pattern='\s+|(&nbsp;)+'
    html=re.sub(pattern,' ',html)
    # 去掉html标记间的空格
    # pattern='> <'
    # html=re.sub(pattern,'><',html)
    html=html.replace('> <','><')
    # html=html.replace('&reg;','®')
    # 去掉object标签和多余的商品行
    pattern='<tr><td valign="top"><table width="99%" height="44" border="0" align="right" cellpadding="0" cellspacing="0"><tr><td></td></tr></table></td></tr>|<object.+?</object>'
    html=re.sub(pattern,'',html)
    # 通过pyquery来规格化html，同时去掉了meta标签
    # return pq(html).outerHtml()
    return html

def findgoods(d):
    trs=d('table[height="233"]>tr')
    return trs.map(lambda i,e:pq(e).outerHtml())
    
def checkcdempty(d):
    return
def checkcdsample(d):
    if d('font[size=4]').text()==u'样 本':
        return True
def mergelog():
    return
def getmergelog():
    return
def mergecd(fname):
    if os.path.isfile('%s.m'%fname):
        if checkcdsample(pq(open('%s.m'%fname).read())):
            os.rename('%s.m'%fname,'%s.s'%fname)
            open('merge_err.log','a').write('%s,delsmp\n'%fname)
        return
    bMerge=False
    if not os.path.isfile(fname):
        open('merge_err.log','a').write('%s,fnf\n'%fname)
    else:
        html=open(fname,'r').read()
        html=htmlclean(html)
        n=d=pq(html)
        if checkcdsample(d):
            open(fname+'.s','w').write(d.outerHtml().encode('utf-8'))
            open('merge_err.log','a').write('%s,smp\n'%fname)
            return
        # print str(d)
        # f=open(fname+'.mp','w')
        # f.write(d.outerHtml().encode('gbk'))
        # f.close()
        table=d('table[height="233"]')
        # print d('div[style="position:absolute; width:142px; height:15px; z-index:2; left: 331px; top: 196px;"]>div').text()
        sign=1
        goodslen=len(findgoods(n))
        if goodslen>0:bMerge=True
        print fname,'goodslen',goodslen

        while goodslen==5:
            sign+=1;
            fn='%s.%s'%(fname,sign)
            # print fn
            if not os.path.isfile(fn):
                open('merge_err.log','a').write('%s,fnf\n'%fn)
                break
            else:
                hn=open(fn).read()
                hn=htmlclean(hn)
                gn=findgoods(pq(hn))
                goodslen=len(gn)
                print fn,'goodslen',goodslen
                # table.append(n('table[height="233"]').html())
                table.append(''.join(gn))
                # f=open('%s.mn%s'%(fname,i),'w')
                # f.write(d.outerHtml().encode('gbk'))
                # f.close()
        if bMerge:
            open(fname+'.m','w').write(d.outerHtml().encode('utf-8'))
        else:
            open(fname+'.b','w').write(d.outerHtml().encode('utf-8'))

def listdir(root,level=1):
    for dirs in os.listdir(root):
        path=os.path.join(root,dirs)
        if level==4:
            cds.append(path)
        if os.path.isdir(path):
            listdir(path,level+1)
def getcdfilerange(dir):
    cd=dir.replace(os.sep,'').split('-')
    s=int(cd[0])
    e=int(cd[0][:11]+cd[1])
    if e>s:
        return s,e+1
    else:
        return e,s+1
def getcdpath(dir):
    # return dir.split('-')[0][:-7]
    return (os.sep).join(dir.split(os.sep)[:4])
    
def company(cd):
    html=open(cd,'r').read()
    html=htmlclean(html)
    pattern=r'<div id="Layer2" style="position:absolute; width:65px; height:15px; z-index:2; left: 233px; top: 251px;"><DIV ALIGN="CENTER">(.+?)</DIV></div>'
    cd_cif=re.search(pattern,html).group(1).decode('gb18030')
    if cd_cif=='CIF':
        pattern=r'<div id="Layer2" style="position:absolute; width:191px; height:15px; z-index:2; left: 380px; top: 164px;">(.+?)</div>'
        cd_voyage=re.search(pattern,html).group(1).decode('gb18030').split('/')[0]
        # print u"船名航次：",cd_voyage
        if cif_voyage(cd_voyage):

            pattern=r'<div id="Layer2" style="position:absolute; width:232px; height:18px; z-index:2; left: 80px; top: 154px;">(.+?)</div>'
            cd_company=re.search(pattern,html).group(1).decode('gb18030')
            # print u"企业名称：",cd_company
            print u"企业名称：",cd_company,u"船名航次：",cd_voyage,cd_cif
            conn.execute("replace into cif_vc (v,c,cif) values(?,?,?)",(cd_voyage,cd_company,cd_cif))

#    if cd_voyage:
#        conn.execute("replace into vc (v,c) values(?,?)",(cd_voyage,cd_company))
    
    
def progress(path,level=1):
    cds=[]
    if level==5:
        cds.append(path)
    else:
        listdir(path,level-1)
    print 'cds',cds
    for c in cds:
        cdpath=getcdpath(c)
        cdfilerange=getcdfilerange(c)
        print cdpath,cdfilerange
        cdrange=cdfilerange[1]-cdfilerange[0]+1
        print cdrange
        for cdfile in range(cdfilerange[0],cdfilerange[1]):
            cd=c+os.sep+str(cdfile)
            # if os.path.isfile(cd):
            # print cd
            # mergecd(cd)
            company(cd)


def cif_voyage(v):
    voyage=('CHUANHE','COSCOHAMBURG','COSCODALIAN','COSCOROTTERDAM','CSCLNINGBO','EVERUNIFIC','EYERUTILE','HANJINKAOHSIUNG','HANJINOSAKA','HANJINPORTLAND ','ITALLAGUNA','OPT','SANTAPRISCILLA','SEASPANNINGBO ','XINCHANGSHU','XINNANSHA','YMOAKLAND','YMSEATTLE')
    if v in voyage:
        return True


# path=('3104','2011','0','54','9900000-9999999')
# path=('3104','2011','0','54','9800000-9899999')
# path=('3104','2011','0','54','9700000-9799999')
paths=[('3104','2011','0','54','9900000-9999999'),('3104','2011','0','54','9800000-9899999'),('3104','2011','0','54','9700000-9799999'),('3104','2011','0','54','9600000-9699999'),('3104','2011','0','54','9500000-9599999'),('3104','2011','0','54','9400000-9499999'),('3104','2011','0','54','9300000-9399999'),('3104','2011','0','54','9200000-9299999'),('3104','2011','0','54','9100000-9199999'),('3104','2011','0','54','9000000-9099999'),('3104','2011','0','54','8900000-8999999'),('3104','2011','0','54','8800000-8899999')]
for path in paths:
    progress((os.sep).join(path),level=len(path))
# fname='310120110517964514'
# mergecd(fname)
conn.commit()
conn.close()    





        